#访问 www.badwe.com 搜索获取信息
import urllib.request as ur
import urllib.parse as up
import lxml.etree as le
import re

url ='https://www.badwe.com/page/1/?s=%E7%BE%8E%E9%A3%9F' #百味生活网搜索页面
def getResponse(url): #获取网页内容
    req = ur.Request(
        url=url,
        headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
        }
    )
    response = ur.urlopen(req).read()
    return response

if __name__ == '__main__':
    keyword = input('请输入关键字') #例如： 美食
    qishi = input('请输入起始页面')
    jieshu = input('请输入结束页面')
    keyword =up.quote(keyword)
    for page in range(int(qishi),int(jieshu)+1):
        print( 'https://www.badwe.com/page/{page}/?s={keyword}'.format(page = page,keyword = keyword))
        response = getResponse(url = 'https://www.badwe.com/page/{page}/?s={keyword}'.format(page = page,keyword = keyword))
        hrefs = le.HTML(response).xpath("//div[@class='item-content']/h2/a/@href")
        for href in hrefs:
            response_content = getResponse(href)
            title = le.HTML(response_content).xpath('//h1/text()')[0]
            title = re.sub(
                r'[/\\:*"<>|?]', '', title
            )
            filepath ="html/%s.html" % title
            with open(filepath,'wb') as f:
                f.write(response_content)
            print(title)

